{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-22T17:29:53.312902Z",
     "iopub.status.busy": "2022-06-22T17:29:53.312671Z",
     "iopub.status.idle": "2022-06-22T17:30:17.944284Z",
     "shell.execute_reply": "2022-06-22T17:30:17.943774Z",
     "shell.execute_reply.started": "2022-06-22T17:29:53.312877Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b6c3ae15ae24d49b705c9f31942bc3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Spark application\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n<tbody><tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr><tr><td>3</td><td>application_1655337734387_0004</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-172-31-1-148.ec2.internal:20888/proxy/application_1655337734387_0004/\" class=\"emr-proxy-link j-8LEPCC9QXO5L application_1655337734387_0004\" emr-resource=\"j-8LEPCC9QXO5L\n\" application-id=\"application_1655337734387_0004\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-172-31-9-136.ec2.internal:8042/node/containerlogs/container_1655337734387_0004_01_000001/livy\">Link</a></td><td>None</td><td>✔</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparkSession available as 'spark'.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pyspark.sql.session.SparkSession object at 0x7fb80693fc90>"
     ]
    }
   ],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-22T17:30:17.945345Z",
     "iopub.status.busy": "2022-06-22T17:30:17.945180Z",
     "iopub.status.idle": "2022-06-22T17:30:27.252063Z",
     "shell.execute_reply": "2022-06-22T17:30:27.251540Z",
     "shell.execute_reply.started": "2022-06-22T17:30:17.945324Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d372dc98e994cc4956f65843ec49dbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "customer_reviews = spark.read.format('parquet') \\\n",
    "                            .load('s3://dsoaws/parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-22T17:30:27.253231Z",
     "iopub.status.busy": "2022-06-22T17:30:27.253063Z",
     "iopub.status.idle": "2022-06-22T17:30:28.016490Z",
     "shell.execute_reply": "2022-06-22T17:30:28.015603Z",
     "shell.execute_reply.started": "2022-06-22T17:30:27.253211Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4433b9baa8746b79d36b3888fa15696",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/mnt/yarn/usercache/livy/appcache/application_1655337734387_0004/container_1655337734387_0004_01_000001/pyspark.zip/pyspark/sql/dataframe.py:140: FutureWarning: Deprecated in 2.0, use createOrReplaceTempView instead."
     ]
    }
   ],
   "source": [
    "customer_reviews.registerTempTable('customer_reviews')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-22T17:30:28.018043Z",
     "iopub.status.busy": "2022-06-22T17:30:28.017777Z",
     "iopub.status.idle": "2022-06-22T17:34:02.053291Z",
     "shell.execute_reply": "2022-06-22T17:34:02.052787Z",
     "shell.execute_reply.started": "2022-06-22T17:30:28.018008Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abda0c05d8954fee93956fa8097b2a3b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+-----------+--------------+----------+--------------+--------------------+-----------+-------------+-----------+----+-----------------+---------------+--------------------+-----------+----+----------------+\n",
      "|marketplace|customer_id|     review_id|product_id|product_parent|       product_title|star_rating|helpful_votes|total_votes|vine|verified_purchase|review_headline|         review_body|review_date|year|product_category|\n",
      "+-----------+-----------+--------------+----------+--------------+--------------------+-----------+-------------+-----------+----+-----------------+---------------+--------------------+-----------+----+----------------+\n",
      "|         FR|   17853693|R3RYA47SSO16FD|B000089RV6|     473856475|Back in Black [Re...|          5|            0|          0|   N|                Y|             cd|livraison rapide ...| 2014-11-17|2014|           Music|\n",
      "+-----------+-----------+--------------+----------+--------------+--------------------+-----------+-------------+-----------+----+-----------------+---------------+--------------------+-----------+----+----------------+"
     ]
    }
   ],
   "source": [
    "one_record = spark.sql('select * from customer_reviews limit 1')\n",
    "one_record.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "python",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
